





<!DOCTYPE html>
<html class="writer-html5" lang="zh-CN" >
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>Putting the VM in TVM: The Relay Virtual Machine &mdash; tvm 0.8.dev1982 文档</title>
  

  
  <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css" integrity="sha384-Gn5384xqQ1aoWXA+058RXPxPg6fy4IWvTNh0E263XmFcJlSAwiGgFAW/dAiS6JXm" crossorigin="anonymous">
  <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
  <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
  <link rel="stylesheet" href="../_static/gallery.css" type="text/css" />
  <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/tlcpack_theme.css" type="text/css" />

  
  
    <link rel="shortcut icon" href="../_static/tvm-logo-square.png"/>
  

  
  
  
  
    
      <script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script>
        <script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js"></script>
        <script src="../_static/jquery.js"></script>
        <script src="../_static/underscore.js"></script>
        <script src="../_static/doctools.js"></script>
        <script src="../_static/translations.js"></script>
    
    <script type="text/javascript" src="../_static/js/theme.js"></script>

    
    <script type="text/javascript" src="../_static/js/tlcpack_theme.js"></script>
    <link rel="index" title="索引" href="../genindex.html" />
    <link rel="search" title="搜索" href="../search.html" />
    <link rel="next" title="Introduction to Module Serialization" href="introduction_to_module_serialization.html" />
    <link rel="prev" title="Debugger" href="debugger.html" /> 
</head>

<body class="wy-body-for-nav">

   
  <div class="wy-grid-for-nav">
    
    
<header class="header">
    <div class="innercontainer">
      <div class="headerInner d-flex justify-content-between align-items-center">
          <div class="headerLogo">
               <a href="https://tvm.apache.org/"><img src=https://tvm.apache.org/assets/images/logo.svg alt="logo"></a>
          </div>

          <div id="headMenu" class="headerNav">
            <button type="button" id="closeHeadMenu" class="navCloseBtn"><img src="../_static/img/close-icon.svg" alt="Close"></button>
             <ul class="nav">
                <li class="nav-item">
                   <a class="nav-link" href=https://tvm.apache.org/community>Community</a>
                </li>
                <li class="nav-item">
                   <a class="nav-link" href=https://tvm.apache.org/download>Download</a>
                </li>
                <li class="nav-item">
                   <a class="nav-link" href=https://tvm.apache.org/vta>VTA</a>
                </li>
                <li class="nav-item">
                   <a class="nav-link" href=https://tvm.apache.org/blog>Blog</a>
                </li>
                <li class="nav-item">
                   <a class="nav-link" href=https://tvm.apache.org/docs>Docs</a>
                </li>
                <li class="nav-item">
                   <a class="nav-link" href=https://tvmconf.org>Conference</a>
                </li>
                <li class="nav-item">
                   <a class="nav-link" href=https://github.com/apache/tvm/>Github</a>
                </li>
                <li class="nav-item">
                   <a class="nav-link" href=https://tvmchinese.github.io/declaration_zh_CN.html>About-Translators</a>
                </li>
             </ul>
               <div class="responsivetlcdropdown">
                 <button type="button" class="btn-link">
                   ASF
                 </button>
                 <ul>
                     <li>
                       <a href=https://apache.org/>Apache Homepage</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/licenses/>License</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/foundation/sponsorship.html>Sponsorship</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/security/>Security</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/foundation/thanks.html>Thanks</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/events/current-event>Events</a>
                     </li>
                     <li>
                       <a href=https://www.zhihu.com/column/c_1429578595417563136>Zhihu</a>
                     </li>
                 </ul>
               </div>
          </div>
            <div class="responsiveMenuIcon">
              <button type="button" id="menuBtn" class="btn-menu"><img src="../_static/img/menu-icon.svg" alt="Menu Icon"></button>
            </div>

            <div class="tlcDropdown">
              <div class="dropdown">
                <button type="button" class="btn-link dropdown-toggle" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
                  ASF
                </button>
                <div class="dropdown-menu dropdown-menu-right">
                  <ul>
                     <li>
                       <a href=https://apache.org/>Apache Homepage</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/licenses/>License</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/foundation/sponsorship.html>Sponsorship</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/security/>Security</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/foundation/thanks.html>Thanks</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/events/current-event>Events</a>
                     </li>
                     <li>
                       <a href=https://www.zhihu.com/column/c_1429578595417563136>Zhihu</a>
                     </li>
                  </ul>
                </div>
              </div>
          </div>
       </div>
    </div>
 </header>
 
    <nav data-toggle="wy-nav-shift" class="wy-nav-side fixed">
      <div class="wy-side-scroll">
        <div class="wy-side-nav-search" >
          

          
            <a href="../index.html">
          

          
            
            <img src="../_static/tvm-logo-small.png" class="logo" alt="Logo"/>
          
          </a>

          
            
            
                <div class="version">
                  0.8.dev1982
                </div>
            
          

          
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
    <input type="text" name="q" placeholder="Search docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

          
        </div>

        
        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          
            
            
              
            
            
              <p class="caption" role="heading"><span class="caption-text">如何开始</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../install/index.html">安装 TVM</a></li>
<li class="toctree-l1"><a class="reference internal" href="../contribute/index.html">贡献者指南</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">用户引导</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../tutorial/index.html">User Tutorial</a></li>
<li class="toctree-l1"><a class="reference internal" href="../how_to/index.html">How To Guides</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">开发者引导</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../dev/tutorial/index.html">Developer Tutorial</a></li>
<li class="toctree-l1"><a class="reference internal" href="../dev/how_to/how_to.html">开发者指南</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">架构指南</span></p>
<ul class="current">
<li class="toctree-l1 current"><a class="reference internal" href="index.html">Design and Architecture</a><ul class="current">
<li class="toctree-l2"><a class="reference internal" href="index.html#example-compilation-flow">编译流程示例</a></li>
<li class="toctree-l2"><a class="reference internal" href="index.html#logical-architecture-components">逻辑架构组件</a></li>
<li class="toctree-l2"><a class="reference internal" href="index.html#tvm-support">tvm/support</a></li>
<li class="toctree-l2 current"><a class="reference internal" href="index.html#tvm-runtime">tvm/runtime</a><ul class="current">
<li class="toctree-l3"><a class="reference internal" href="runtime.html">TVM runtime系统</a></li>
<li class="toctree-l3"><a class="reference internal" href="runtime.html#runtime-specific-information">Runtime-Specific Information</a></li>
<li class="toctree-l3"><a class="reference internal" href="debugger.html">Debugger</a></li>
<li class="toctree-l3 current"><a class="current reference internal" href="#">Putting the VM in TVM: The Relay Virtual Machine</a><ul>
<li class="toctree-l4"><a class="reference internal" href="#design">Design</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="introduction_to_module_serialization.html">Introduction to Module Serialization</a></li>
<li class="toctree-l3"><a class="reference internal" href="device_target_interactions.html">Device/Target Interactions</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="index.html#tvm-node">tvm/node</a></li>
<li class="toctree-l2"><a class="reference internal" href="index.html#tvm-ir">tvm/ir</a></li>
<li class="toctree-l2"><a class="reference internal" href="index.html#tvm-target">tvm/target</a></li>
<li class="toctree-l2"><a class="reference internal" href="index.html#tvm-tir">tvm/tir</a></li>
<li class="toctree-l2"><a class="reference internal" href="index.html#tvm-arith">tvm/arith</a></li>
<li class="toctree-l2"><a class="reference internal" href="index.html#tvm-te">tvm/te</a></li>
<li class="toctree-l2"><a class="reference internal" href="index.html#tvm-topi">tvm/topi</a></li>
<li class="toctree-l2"><a class="reference internal" href="index.html#tvm-relay">tvm/relay</a></li>
<li class="toctree-l2"><a class="reference internal" href="index.html#tvm-autotvm">tvm/autotvm</a></li>
<li class="toctree-l2"><a class="reference internal" href="index.html#frontends">Frontends</a></li>
<li class="toctree-l2"><a class="reference internal" href="index.html#security">安全</a></li>
<li class="toctree-l2"><a class="reference internal" href="index.html#microtvm">microTVM</a></li>
</ul>
</li>
</ul>
<p class="caption" role="heading"><span class="caption-text">主题引导</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../topic/microtvm/index.html">microTVM：裸机使用TVM</a></li>
<li class="toctree-l1"><a class="reference internal" href="../topic/vta/index.html">VTA: Versatile Tensor Accelerator</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">参考指南</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../reference/langref/index.html">语言参考</a></li>
<li class="toctree-l1"><a class="reference internal" href="../reference/api/python/index.html">Python API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../reference/api/links.html">Other APIs</a></li>
<li class="toctree-l1"><a class="reference internal" href="../reference/publications.html">Publications</a></li>
<li class="toctree-l1"><a class="reference internal" href="../genindex.html">索引</a></li>
</ul>

            
          
        </div>
        
      </div>
    </nav>

    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
      
      <nav class="wy-nav-top" aria-label="top navigation" data-toggle="wy-nav-top">
        
            <div class="togglemenu">

            </div>
            <div class="nav-content">
              <!-- tvm -->
              Table of content
            </div>
        
      </nav>


      <div class="wy-nav-content">
        
        <div class="rst-content">
        

          




















<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="wy-breadcrumbs">
    
      <li><a href="../index.html">Docs</a> <span class="br-arrow">></span></li>
        
          <li><a href="index.html">Design and Architecture</a> <span class="br-arrow">></span></li>
        
      <li>Putting the VM in TVM: The Relay Virtual Machine</li>
    
    
      <li class="wy-breadcrumbs-aside">
        
            
            <a href="../_sources/arch/virtual_machine.rst.txt" rel="nofollow"> <img src="../_static//img/source.svg" alt="viewsource"/></a>
          
        
      </li>
    
  </ul>

  
  <hr/>
</div>
          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
           <div itemprop="articleBody">
            
  <div class="section" id="putting-the-vm-in-tvm-the-relay-virtual-machine">
<h1>Putting the VM in TVM: The Relay Virtual Machine<a class="headerlink" href="#putting-the-vm-in-tvm-the-relay-virtual-machine" title="永久链接至标题">¶</a></h1>
<p>Relay, a new program representation, has enabled the representation and optimization of
a great breadth of machine learning programs.
Unfortunately, by supporting a more expressive set of programs, we have
introduced several new execution challenges.</p>
<p>Relay’s interpreter can execute the full language but has notable limitations
that make it unsuited for production deployments. It is structured as an inefficient
interpreter that performs AST traversal to execute the program. This approach is conceptually
simple but inefficient, as the AST traversal heavily relies on indirection.</p>
<p>There are further challenges in compiling dynamic code, such as dynamic scheduling and allocation,
fully dynamic tensor shapes, and control flow. The interpreter offers simple solutions
for these, but none is sufficiently compelling or optimized.</p>
<p>The second execution mechanism is the existing graph executor. In order to target Relay
programs to this, we compile a small subset of them to the old graph format and execute
them on the runtime. Graph executor provides a fast execution experience but only for a very limited
subset of Relay programs.</p>
<p>An alternative but not-standard approach is Relay’s ahead-of-time compiler,
which compiles a Relay program into a shared library containing an ahead-of-time
implementation. The ahead-of-time compiler provides compelling performance
but is difficult to extend and instrument, which can only be done by modifying the
code generation and optimization mechanisms.</p>
<p>The Relay virtual machine is intended to be a framework that balances these competing
approaches, providing a dynamic execution environment which can be extended, instrumented,
and integrated with other approaches like ahead-of-time compilation via a flexible extension
mechanism.</p>
<p>The virtual machine is designed to strike a balance between performance and flexibility
when deploying and executing Relay programs, without giving up the benefits of TVM.</p>
<p>Virtual machine (VM) design is a well-studied area in programming languages and systems,
and there have been various virtual machine designs for both full-fledged
and embedded programing languages.
Previous language VM designs have been heavily tailored to the execution profile of traditional programs.
Traditional programs manipulate small scalar values and consist of a large number of low-level instructions.
The sheer quantity of instructions requires instruction execution and dispatch to be extremely efficient.
In the context of machine learning we manipulate primarily tensor values, using a (relatively)
low number of high level instructions. ML programs’ cost centers are expensive operator invocations,
such as GEMM or convolution, over a large input. Due to the execution profile exhibited by ML programs,
micro-optimizations present in scalar VMs are dramatically less important.</p>
<p>TVM has provided strong support for vision models,
but we want to grow to support a wider variety of models.
The graph executor is able to utilize the fully static nature of the input graphs to perform
aggressive optimization such as fully static allocation, and optimal memory reuse.
When we introduce models which make use of control flow, recursion, dynamic shapes, and dynamic
allocation, we must change how execution works. A virtual machine for Relay is a natural choice.</p>
<p>The rest of this document provides a high-level overview of the Relay
virtual machine design and its instruction set.</p>
<div class="section" id="design">
<h2>Design<a class="headerlink" href="#design" title="永久链接至标题">¶</a></h2>
<p>The VM’s design is focused on simplicity without sacrificing performance.
In order to accomplish this we have focused on designing a tensor VM rather than a scalar VM.</p>
<p>In the tensor VM setting, we optimize for cheap “allocation” of objects (by trying to avoid real allocation),
reuse of static fragments, and the ability to do dynamic shape (i.e jagged tensors).</p>
<div class="section" id="instruction-set">
<h3>Instruction Set<a class="headerlink" href="#instruction-set" title="永久链接至标题">¶</a></h3>
<p>The choices of an instruction set and instruction representation are the most critical design decisions for a VM.
The current representation of the instructions is a tagged union containing the op-code and the data payload.  An important design decision is the level of abstraction of the instructions (RISC vs. CISC) and how they take their data (fixed-width instruction encoding vs. variable-length encoding). The current version is closer to CISC, with complex instructions like AllocTensor, and is variable-length due to the inclusion of the shape as part of the instruction. The current instruction set is very high-level and corresponds roughly to high-level operations in Relay.</p>
<div class="section" id="ret">
<h4>Ret<a class="headerlink" href="#ret" title="永久链接至标题">¶</a></h4>
<p><strong>Arguments</strong>:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">RegName</span> <span class="n">dst</span>
<span class="n">RegName</span> <span class="n">result</span>
</pre></div>
</div>
<p>Returns the object in register <code class="docutils literal notranslate"><span class="pre">result</span></code> to caller’s register <code class="docutils literal notranslate"><span class="pre">dst</span></code>.</p>
</div>
<div class="section" id="invokepacked">
<h4>InvokePacked<a class="headerlink" href="#invokepacked" title="永久链接至标题">¶</a></h4>
<p><strong>Arguments</strong>:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">Index</span> <span class="n">packed_index</span>
<span class="n">Index</span> <span class="n">arity</span>
<span class="n">Index</span> <span class="n">output_size</span>
<span class="n">RegName</span><span class="o">*</span> <span class="n">packed_args</span>
</pre></div>
</div>
<p>Invoke the packed function denoted by <code class="docutils literal notranslate"><span class="pre">packed_index</span></code>. The <code class="docutils literal notranslate"><span class="pre">arity</span></code>
and <code class="docutils literal notranslate"><span class="pre">output_size</span></code> are used to inform the VM how many inputs and
outputs to expect. <code class="docutils literal notranslate"><span class="pre">packed_args</span></code> stores the list of argument registers. Note <code class="docutils literal notranslate"><span class="pre">Index</span></code>
is an alias of <code class="docutils literal notranslate"><span class="pre">int64_t</span></code>, and it will be used in other instructions as well.</p>
</div>
<div class="section" id="alloctensor">
<h4>AllocTensor<a class="headerlink" href="#alloctensor" title="永久链接至标题">¶</a></h4>
<p><strong>Arguments</strong>:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">RegName</span> <span class="n">dst</span>
<span class="n">RegName</span> <span class="n">storage</span>
<span class="n">uint32_t</span> <span class="n">ndim</span>
<span class="n">int64_t</span><span class="o">*</span> <span class="n">shape</span>
<span class="n">DLDataType</span> <span class="n">dtype</span>
</pre></div>
</div>
<p>Allocate a tensor value of using constant shape (stored in <code class="docutils literal notranslate"><span class="pre">shape</span></code>) and <code class="docutils literal notranslate"><span class="pre">dtype</span></code>
from the given storage block, <code class="docutils literal notranslate"><span class="pre">storage</span></code>. The result is saved to register <code class="docutils literal notranslate"><span class="pre">dst</span></code>.</p>
</div>
<div class="section" id="alloctensorreg">
<h4>AllocTensorReg<a class="headerlink" href="#alloctensorreg" title="永久链接至标题">¶</a></h4>
<p><strong>Arguments</strong>:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">RegName</span> <span class="n">dst</span>
<span class="n">RegName</span> <span class="n">storage</span>
<span class="n">RegName</span> <span class="n">shape_register</span>
<span class="n">DLDataType</span> <span class="n">dtype</span>
</pre></div>
</div>
<p>Allocate a tensor value of the appropriate shape (stored in <code class="docutils literal notranslate"><span class="pre">shape_register</span></code>)
and <code class="docutils literal notranslate"><span class="pre">dtype</span></code> from the given storage block (stored in <code class="docutils literal notranslate"><span class="pre">storage</span></code>). The result is saved to register <code class="docutils literal notranslate"><span class="pre">dst</span></code>.</p>
</div>
<div class="section" id="allocstorage">
<h4>AllocStorage<a class="headerlink" href="#allocstorage" title="永久链接至标题">¶</a></h4>
<p><strong>Arguments</strong>:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">RegName</span> <span class="n">dst</span>
<span class="n">RegName</span> <span class="n">size</span>
<span class="n">RegName</span> <span class="n">alignment</span>
<span class="n">DLDataType</span> <span class="n">dtype_hint</span>
</pre></div>
</div>
<p>Allocate a storage block with the given <code class="docutils literal notranslate"><span class="pre">size</span></code>, <code class="docutils literal notranslate"><span class="pre">alignment</span></code> and data type, <code class="docutils literal notranslate"><span class="pre">dtype_hint</span></code>.
The allocated storage block is stored in register <code class="docutils literal notranslate"><span class="pre">dst</span></code>.</p>
</div>
<div class="section" id="allocadt">
<h4>AllocADT<a class="headerlink" href="#allocadt" title="永久链接至标题">¶</a></h4>
<p><strong>Arguments</strong>:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">RegName</span> <span class="n">dst</span>
<span class="n">Index</span> <span class="n">tag</span>
<span class="n">Index</span> <span class="n">num_fields</span>
<span class="n">RegName</span><span class="o">*</span> <span class="n">datatype_fields</span>
</pre></div>
</div>
<p>Allocate a data type with the tag <code class="docutils literal notranslate"><span class="pre">tag</span></code> using the <code class="docutils literal notranslate"><span class="pre">num_fields</span></code> entries
from registers <code class="docutils literal notranslate"><span class="pre">datatype_fields</span></code>. The result is saved to register <code class="docutils literal notranslate"><span class="pre">dst</span></code>.</p>
</div>
<div class="section" id="allocclosure">
<h4>AllocClosure<a class="headerlink" href="#allocclosure" title="永久链接至标题">¶</a></h4>
<p><strong>Arguments</strong>:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">RegName</span> <span class="n">dst</span>
<span class="n">Index</span> <span class="n">clo_index</span>
<span class="n">Index</span> <span class="n">num_freevar</span>
<span class="n">RegName</span><span class="o">*</span> <span class="n">free_vars</span><span class="p">;</span>
</pre></div>
</div>
<p>Allocate a closure with the VMFunction at <code class="docutils literal notranslate"><span class="pre">clo_index</span></code> as
its code, and the <code class="docutils literal notranslate"><span class="pre">num_freevar</span></code> entries from registers in
<code class="docutils literal notranslate"><span class="pre">free_vars</span></code>. The result is saved to register <code class="docutils literal notranslate"><span class="pre">dst</span></code>.</p>
</div>
<div class="section" id="getfield">
<h4>GetField<a class="headerlink" href="#getfield" title="永久链接至标题">¶</a></h4>
<p><strong>Arguments</strong>:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">RegName</span> <span class="n">dst</span>
<span class="n">RegName</span> <span class="nb">object</span>
<span class="n">Index</span> <span class="n">field_index</span>
</pre></div>
</div>
<p>Get the field value with index <code class="docutils literal notranslate"><span class="pre">field_index</span></code> from <code class="docutils literal notranslate"><span class="pre">object</span></code>. And saves the result to register <code class="docutils literal notranslate"><span class="pre">dst</span></code>.</p>
</div>
<div class="section" id="if">
<h4>If<a class="headerlink" href="#if" title="永久链接至标题">¶</a></h4>
<p><strong>Arguments</strong>:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">RegName</span> <span class="n">test</span>
<span class="n">RegName</span> <span class="n">target</span>
<span class="n">Index</span> <span class="n">true_offset</span>
<span class="n">Index</span> <span class="n">false_offset</span>
</pre></div>
</div>
<p>Check if the object at register <code class="docutils literal notranslate"><span class="pre">test</span></code> is equal to <code class="docutils literal notranslate"><span class="pre">target</span></code>.
If equal, relative jump by <code class="docutils literal notranslate"><span class="pre">true_offset</span></code>, else relative
jump by <code class="docutils literal notranslate"><span class="pre">false_offset</span></code>.</p>
</div>
<div class="section" id="gettag">
<h4>GetTag<a class="headerlink" href="#gettag" title="永久链接至标题">¶</a></h4>
<p><strong>Arguments</strong>:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">RegName</span> <span class="nb">object</span>
<span class="n">RegName</span> <span class="n">dst</span>
</pre></div>
</div>
<p>Get the object tag for ADT object in register <code class="docutils literal notranslate"><span class="pre">object</span></code>. And saves the reult to register <code class="docutils literal notranslate"><span class="pre">dst</span></code>.</p>
</div>
<div class="section" id="fatal">
<h4>Fatal<a class="headerlink" href="#fatal" title="永久链接至标题">¶</a></h4>
<p>Fail the virtual machine execution.</p>
</div>
<div class="section" id="goto">
<h4>Goto<a class="headerlink" href="#goto" title="永久链接至标题">¶</a></h4>
<p><strong>Arguments</strong>:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">Index</span> <span class="n">pc_offset</span>
</pre></div>
</div>
<p>Relative unconditional jump by <code class="docutils literal notranslate"><span class="pre">pc_offset</span></code>.</p>
</div>
<div class="section" id="invoke">
<h4>Invoke<a class="headerlink" href="#invoke" title="永久链接至标题">¶</a></h4>
<p><strong>Arguments</strong>:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">Index</span> <span class="n">func_index</span>
</pre></div>
</div>
<p>Invoke function at <code class="docutils literal notranslate"><span class="pre">func_index</span></code>, consumes the number of arguments contained in the VMFunction’s
arity field.</p>
</div>
<div class="section" id="invokeclosure">
<h4>InvokeClosure<a class="headerlink" href="#invokeclosure" title="永久链接至标题">¶</a></h4>
<p><strong>Arguments</strong>:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">RegName</span> <span class="n">closure</span>
<span class="n">Index</span> <span class="n">num_closure_args</span>
<span class="n">RegName</span><span class="o">*</span> <span class="n">closure_args</span>
</pre></div>
</div>
<p>Invokes <code class="docutils literal notranslate"><span class="pre">closure</span></code>, consuming the number of arguments declared in the closure’s VMFunction.</p>
</div>
<div class="section" id="loadconst">
<h4>LoadConst<a class="headerlink" href="#loadconst" title="永久链接至标题">¶</a></h4>
<p><strong>Arguments</strong>:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">RegName</span> <span class="n">dst</span>
<span class="n">Index</span> <span class="n">const_index</span>
</pre></div>
</div>
<p>Load the constant at <code class="docutils literal notranslate"><span class="pre">const_index</span></code> from the constant pool. The result is saved to register <code class="docutils literal notranslate"><span class="pre">dst</span></code>.</p>
</div>
<div class="section" id="loadconsti">
<h4>LoadConsti<a class="headerlink" href="#loadconsti" title="永久链接至标题">¶</a></h4>
<p><strong>Arguments</strong>:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">Index</span> <span class="n">val</span>
<span class="n">RegName</span> <span class="n">dst</span>
</pre></div>
</div>
<p>Load the constant integer <code class="docutils literal notranslate"><span class="pre">val</span></code> to register <code class="docutils literal notranslate"><span class="pre">dst</span></code>. The result is a 0-rank tensor.</p>
</div>
</div>
<div class="section" id="object-representation">
<h3>Object Representation<a class="headerlink" href="#object-representation" title="永久链接至标题">¶</a></h3>
<p>We leverage the object protocol to represent the objects that are used by the
VM.</p>
<p>Currently, three types of objects, <code class="docutils literal notranslate"><span class="pre">NDArray</span></code>, <code class="docutils literal notranslate"><span class="pre">ADT</span></code>, and <code class="docutils literal notranslate"><span class="pre">Closure</span></code> objects, are used
to represent tensor, tuple/list, and closure data, respectively. More details
for each of them can be found at <a class="reference external" href="https://github.com/apache/tvm/blob/main/include/tvm/runtime/ndarray.h">include/tvm/runtime/ndarray.h</a>,
<a class="reference external" href="https://github.com/apache/tvm/blob/main/include/tvm/runtime/vm/vm.h">include/tvm/runtime/vm/vm.h</a>, and <a class="reference external" href="https://github.com/apache/tvm/blob/main/include/tvm/runtime/container.h">include/tvm/runtime/container.h</a>, respectively.</p>
</div>
<div class="section" id="stack-and-state">
<h3>Stack and State<a class="headerlink" href="#stack-and-state" title="永久链接至标题">¶</a></h3>
<p>The Relay VM maintains a stack frame, which contains information about how to resume the
previous call. Registers are allocated in a continuous space (virtual register file) for each function.</p>
<p>We keep track of a set of Relay functions we have called, a pointer into its bytecode, an offset into the byte code (known as the program counter).</p>
<div class="highlight-c notranslate"><div class="highlight"><pre><span></span><span class="k">struct</span> <span class="n">VirtualMachine</span> <span class="p">{</span>
  <span class="p">...</span>
  <span class="n">std</span><span class="o">::</span><span class="n">vector</span><span class="o">&lt;</span><span class="n">VMFrame</span><span class="o">&gt;</span> <span class="n">frames</span><span class="p">;</span>
  <span class="p">...</span>
  <span class="c1">// Current function.</span>
  <span class="kt">size_t</span> <span class="n">func_index</span><span class="p">;</span>
  <span class="c1">// Pointer into the current function&#39;s instructions.</span>
  <span class="k">const</span> <span class="n">Instruction</span><span class="o">*</span> <span class="n">code</span><span class="p">;</span>
  <span class="c1">// Current program counter relative to the code pointer.</span>
  <span class="kt">size_t</span> <span class="n">pc</span><span class="p">;</span>
  <span class="p">...</span>
<span class="p">};</span>
</pre></div>
</div>
</div>
<div class="section" id="dispatch-loop">
<h3>Dispatch Loop<a class="headerlink" href="#dispatch-loop" title="永久链接至标题">¶</a></h3>
<p>A critical piece of a VM is the dispatch loop. The dispatch loop usually dominates the execution time of a
virtual machine, but we have experimentally found this not to be the case for Relay. We have just implemented
a simple <code class="docutils literal notranslate"><span class="pre">switch</span></code>/<code class="docutils literal notranslate"><span class="pre">goto</span></code> dispatch loop which dispatches based on instruction op code.</p>
<p>This loop is implemented by <code class="docutils literal notranslate"><span class="pre">VirtualMachine::Run()</span></code>.</p>
</div>
<div class="section" id="vm-compiler">
<h3>VM Compiler<a class="headerlink" href="#vm-compiler" title="永久链接至标题">¶</a></h3>
<p>An important part of this infrastructure is a compiler from Relay’s full IR into a sequence of bytecode.
The VM compiler transforms a <code class="docutils literal notranslate"><span class="pre">tvm::relay::Module</span></code> into a <code class="docutils literal notranslate"><span class="pre">tvm::relay::vm::Executable</span></code>. The executable
contains a set of compiled functions, the compiled functions are contained in <code class="docutils literal notranslate"><span class="pre">tvm::relay::vm::Function</span></code>.
The functions contain metadata about the function as well as its compiled bytecode. The emitted executable
object then can be loaded and run by a <code class="docutils literal notranslate"><span class="pre">tvm::relay::vm::VirtualMachine</span></code> object. For full definitions of the
data structures, please see <a class="reference external" href="https://github.com/apache/tvm/blob/main/include/tvm/runtime/vm/executable.h">include/tvm/runtime/vm/executable.h</a> and <a class="reference external" href="https://github.com/apache/tvm/blob/main/include/tvm/runtime/vm/vm.h">include/tvm/runtime/vm/vm.h</a>.</p>
</div>
<div class="section" id="optimizations">
<h3>Optimizations<a class="headerlink" href="#optimizations" title="永久链接至标题">¶</a></h3>
<p>There are quite a few optimizations required by the VM compiler. Each of them
is implemented as a pass which is managed by the Relay pass manager.</p>
<p>Optimizations marked with <cite>TODO</cite> are not implemented yet.</p>
<ul class="simple">
<li><p>A-Normal Form</p></li>
<li><p>Lambda Lift (see <a class="reference external" href="https://github.com/apache/tvm/blob/main/src/relay/backend/vm/lambda_lift.cc">src/relay/vm/lambda_lift.cc</a>)</p></li>
<li><p>Inline Primitives (see <a class="reference external" href="https://github.com/apache/tvm/blob/main/src/relay/backend/vm/inline_primitives.cc">src/relay/vm/inline_primitives.cc</a>)</p></li>
<li><p>Constant Pool Layout (see <a class="reference external" href="https://github.com/apache/tvm/blob/main/src/relay/backend/vm/compiler.cc">src/relay/backend/vm/compiler.cc</a>)</p></li>
<li><p>Tail Call Optimization (TODO)</p></li>
<li><p>Liveness Analysis (TODO)</p></li>
</ul>
</div>
<div class="section" id="serialization">
<h3>Serialization<a class="headerlink" href="#serialization" title="永久链接至标题">¶</a></h3>
<p>Serializing and deserializing the executable generated by the Relay VM compiler is a must as
we may want to save the model to the disk and perform inference later. Previously, Relay has produced
a serialized form in a json file for the graph executor. However, the same format is not directly
applicable to the VM as it emits bytecode instead of graph-style programs.
Serialization of an executable essentially needs to handle both model specific
(i.e. weights and kernels) and VM related (i.e. bytecode and global function names) data.</p>
<p>For kernels, we can conveniently leverage existing TVM infra to save and load
the compiled library module. Here we only focus on serializing other several
components in a binary format that is organized with the following sections in order.</p>
<ul class="simple">
<li><p>Global section. This section contains the globals (function names) used by the virtual machine.</p></li>
<li><p>Constant section. This section is used to store the constant pool (i.e. weights of the model)
for a virtual machine.</p></li>
<li><p>Primitive name section. This section is introduced to accommodate the list of primitive
operator names that will be invoked by the virtual machine, i.e. the names
starting with <code class="docutils literal notranslate"><span class="pre">fused_</span></code>. The primitive names are used as symbols to look up
function pointers in the compiled kernel library.</p></li>
<li><p>Code section. The VM functions, including bytecode, are sitting in this section. The dispatching
loop iterates through this section to fetch instructions for execution.</p></li>
</ul>
<p>Hence, unlike the graph executor artifact that contains weight (.params), graph json (.json),
and compiled kernel library (.so), the serialized executable artifact is composed of the Relay
object file (.ro) and the compiled kernel library (.so).</p>
<p>A <code class="docutils literal notranslate"><span class="pre">save</span></code> function is implemented to store the executable to the disk and
serialize it into the above format. Meanwhile, a <code class="docutils literal notranslate"><span class="pre">load_exec</span></code> function is used to
load the serialized kernel binary and executable related binary code, which will be again used to
instantiate a VM object. Please refer to the <a class="reference external" href="https://github.com/apache/tvm/blob/main/tests/python/relay/test_vm_serialization.py">test_vm_serialization.py</a> file for more
examples.</p>
</div>
<div class="section" id="unresolved-questions">
<h3>Unresolved Questions<a class="headerlink" href="#unresolved-questions" title="永久链接至标题">¶</a></h3>
<div class="section" id="how-do-we-handle-dynamic-shapes">
<h4>How do we handle dynamic shapes?<a class="headerlink" href="#how-do-we-handle-dynamic-shapes" title="永久链接至标题">¶</a></h4>
<p>Dynamic shape support is ongoing work in TVM as we upgrade Relay, TVM’s compiler.  For the most recent updates on
dynamic shape support, we recommend following updates in TVM’s Discuss forum (<a class="reference external" href="https://discuss.tvm.apache.org/">https://discuss.tvm.apache.org/</a>).</p>
</div>
<div class="section" id="how-can-we-modify-the-vm-to-support-jit-compilation-of-certain-code-paths">
<h4>How can we modify the VM to support JIT compilation of certain code paths?<a class="headerlink" href="#how-can-we-modify-the-vm-to-support-jit-compilation-of-certain-code-paths" title="永久链接至标题">¶</a></h4>
<p>In the code generation space there are still many tradeoffs to be analyzed and the VM is designed
to be very flexible so we can modify it for future experiments.</p>
</div>
<div class="section" id="how-do-we-support-heterogenous-execution">
<h4>How do we support heterogenous execution?<a class="headerlink" href="#how-do-we-support-heterogenous-execution" title="永久链接至标题">¶</a></h4>
<p>Heterogenous execution should work out of the box assuming we have annotated the appropriate device copies.
In order to do this properly we need to run the device annotation and copying passes.</p>
</div>
</div>
</div>
</div>


           </div>
           
          </div>
          

<footer>

    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
      
        <a href="introduction_to_module_serialization.html" class="btn btn-neutral float-right" title="Introduction to Module Serialization" accesskey="n" rel="next">下一个 <span class="fa fa-arrow-circle-right"></span></a>
      
      
        <a href="debugger.html" class="btn btn-neutral float-left" title="Debugger" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> 上一个</a>
      
    </div>

<div id="button" class="backtop"><img src="../_static//img/right.svg" alt="backtop"/> </div>
<section class="footerSec">
    <div class="footerHeader">
      <ul class="d-flex align-md-items-center justify-content-between flex-column flex-md-row">
        <li class="copywrite d-flex align-items-center">
          <h5 id="copy-right-info">© 2020 Apache Software Foundation | All right reserved</h5>
        </li>
      </ul>

    </div>

    <ul>
      <li class="footernote">Copyright © 2020 The Apache Software Foundation. Apache TVM, Apache, the Apache feather, and the Apache TVM project logo are either trademarks or registered trademarks of the Apache Software Foundation.</li>
    </ul>

</section>
</footer>
        </div>
      </div>

    </section>

  </div>
  

    <script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.12.9/umd/popper.min.js" integrity="sha384-ApNbgh9B+Y1QKtv3Rn7W3mgPxhU9K/ScQsAP7hUibX39j7fakFPskvXusvfa0b4Q" crossorigin="anonymous"></script>
    <script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/js/bootstrap.min.js" integrity="sha384-JZR6Spejh4U02d8jOt6vLEHfe/JQGiRRSQQxSfFWpi1MquVdAyjUar5+76PVCmYl" crossorigin="anonymous"></script>

  </body>
  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script>

  
  
    
    <!-- Theme Analytics -->
    <script>
    (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
      (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
      m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
    })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');

    ga('create', 'UA-75982049-2', 'auto');
    ga('send', 'pageview');
    </script>

    
   

</body>
</html>